{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self,d_model,head,dropout=0.1):\n",
    "        super().__init__()\n",
    "        self.d_model = d_model\n",
    "        self.d_k = d_model // head\n",
    "        self.h  = head\n",
    "\n",
    "        self.q_linear=nn.Linear(d_model,d_model)\n",
    "        self.k_linear=nn.Linear(d_model,d_model)\n",
    "        self.v_linear=nn.Linear(d_model,d_model)\n",
    "\n",
    "        self.dropout=nn.Dropout(dropout)\n",
    "        self.out=nn.Linear(d_model,d_model)\n",
    "\n",
    "    def attention(q,k,v,d_k,dropout=None,mask=None):\n",
    "        scores = torch.matmul(q,k.transpose(-2,-1)) / math.sqrt(d_k)\n",
    "\n",
    "        if mask:\n",
    "            scores=scores.masked_fill(mask==0,-1e9)\n",
    "\n",
    "        scores = F.softmax(scores,dim=-1)\n",
    "\n",
    "        if dropout:\n",
    "            scores=self.dropout(scores)\n",
    "\n",
    "        output=torch.matmul(scores,v)\n",
    "\n",
    "        return output\n",
    "    \n",
    "    def forward(self,q,k,v):\n",
    "        batch_size=q.size(0)\n",
    "\n",
    "        k=self.k_linear(k).view(bs,-1,self.h,self.d_k)\n",
    "        q=self.q_linear(q).view(bs,-1,self.h,self.d_k)\n",
    "        v=self.v_linear(v).view(bs,-1,self.h,self.d_k)\n",
    "\n",
    "        k=k.transpose(1,2)\n",
    "        q=q.transpose(1,2)\n",
    "        v=v.transpose(1,2)\n",
    "\n",
    "        scores=self.attention(q,k,v,self.d_k,self.dropout)\n",
    "        concat = scores.transpose(1,2).view(batch_size,-1,self.d_model)\n",
    "        output=self.out(concat)\n",
    "\n",
    "        return output\n"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
